{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "    http://www.apache.org/licenses/LICENSE-2.0\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time\n",
    "start = time.time()\n",
    "very_start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask\n",
    "from dask.distributed import Client, wait, LocalCluster\n",
    "import dask.dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>tcp://10.2.48.253:41469</li>\n",
       "  <li><b>Dashboard: </b><a href='http://10.2.48.253:8787/status' target='_blank'>http://10.2.48.253:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>8</li>\n",
       "  <li><b>Cores: </b>8</li>\n",
       "  <li><b>Memory: </b>720.00 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: 'tcp://10.2.48.253:41469' processes=8 threads=8, memory=720.00 GB>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "client = Client(n_workers=8, \n",
    "                       threads_per_worker=1,\n",
    "                       memory_limit='90GB',ip='10.2.48.253')\n",
    "#client = Client(ip='10.2.48.253',memory_limit='100GB')\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 43.3 ms, sys: 3.88 ms, total: 47.1 ms\n",
      "Wall time: 44.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "path = '/raid/data/recsys/train_split'\n",
    "train = dd.read_parquet(f'{path}/train-preproc-fold-*.parquet')#,dtypes=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.9 ms, sys: 365 µs, total: 11.2 ms\n",
      "Wall time: 9.73 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# DROP UNUSED COLUMNS\n",
    "cols_drop = ['links','hashtags0', 'hashtags1', 'fold']\n",
    "train = train.drop(cols_drop,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask.dataframe.core.DataFrame'> (Delayed('int-3ca0acaf-8dfd-4314-a900-5ec2a9d49516'), 25)\n",
      "CPU times: user 1.94 ms, sys: 3.77 ms, total: 5.71 ms\n",
      "Wall time: 4.98 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask.dataframe.core.DataFrame'> (Delayed('int-4af478ea-baac-41c0-a1b5-1294f8225550'), 25)\n",
      "CPU times: user 8.26 ms, sys: 21 µs, total: 8.28 ms\n",
      "Wall time: 7.44 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train = train.repartition(npartitions=8)\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for i,p in enumerate(train.partitions):\n",
    "#    print(i,len(p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = ['reply', 'retweet', 'retweet_comment', 'like']\n",
    "for col in train.columns:\n",
    "    if col in label_names:\n",
    "        train[col] = train[col].astype('float32')\n",
    "    elif train[col].dtype=='int64':\n",
    "        train[col] = train[col].astype('int32')\n",
    "    elif train[col].dtype=='int16':\n",
    "        train[col] = train[col].astype('int8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.8 ms, sys: 332 µs, total: 15.1 ms\n",
      "Wall time: 13.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train = train.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Delayed('int-ab8d5782-7fda-45e1-9870-0004c78683f7'), 25)\n",
      "CPU times: user 11.4 ms, sys: 248 µs, total: 11.7 ms\n",
      "Wall time: 10.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 53.8 ms, sys: 4.4 ms, total: 58.2 ms\n",
      "Wall time: 53.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# TIME FEATURES\n",
    "# RAPIDS does this 5x faster than Pandas CPU\n",
    "# If we didn't need to copy CPU to GPU to CPU, then 1300x faster!\n",
    "def split_time(df):\n",
    "    #gf = cudf.from_pandas(df[['timestamp']])\n",
    "    df['dt_dow']  = df['timestamp'].dt.weekday#.to_array() \n",
    "    df['dt_hour'] = df['timestamp'].dt.hour#.to_array()\n",
    "    df['dt_minute'] = df['timestamp'].dt.minute#.to_array()\n",
    "    df['dt_second'] = df['timestamp'].dt.second#.to_array()\n",
    "    return df\n",
    "\n",
    "train = split_time(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>engage_time</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 09:26:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 18:41:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 01:13:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-02-07 12:36:47</td>\n",
       "      <td>2020-02-07 12:15:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-02-09 13:33:47</td>\n",
       "      <td>2020-02-08 14:14:39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          engage_time           timestamp\n",
       "0 1970-01-01 00:00:00 2020-02-09 09:26:50\n",
       "1 1970-01-01 00:00:00 2020-02-09 18:41:35\n",
       "2 1970-01-01 00:00:00 2020-02-09 01:13:28\n",
       "3 2020-02-07 12:36:47 2020-02-07 12:15:20\n",
       "4 2020-02-09 13:33:47 2020-02-08 14:14:39"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()[['engage_time','timestamp']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('<M8[ns]')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.timestamp.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 35.9 ms, sys: 8.05 ms, total: 43.9 ms\n",
      "Wall time: 40.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# ELAPSED TIME\n",
    "for col in ['engage_time','timestamp']:\n",
    "    train[col] = train[col].astype('int64')/1e9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask.dataframe.core.DataFrame'> (Delayed('int-943b7243-63c6-4fc8-8292-c033b18a8f40'), 29)\n",
      "CPU times: user 15.5 ms, sys: 4.02 ms, total: 19.5 ms\n",
      "Wall time: 18 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>engage_time</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581240e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581274e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581211e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.581079e+09</td>\n",
       "      <td>1.581078e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.581255e+09</td>\n",
       "      <td>1.581171e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    engage_time     timestamp\n",
       "0  0.000000e+00  1.581240e+09\n",
       "1  0.000000e+00  1.581274e+09\n",
       "2  0.000000e+00  1.581211e+09\n",
       "3  1.581079e+09  1.581078e+09\n",
       "4  1.581255e+09  1.581171e+09"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()[['engage_time','timestamp']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_nan(ds):\n",
    "    mask = ds == 0\n",
    "    ds.loc[mask] = np.nan\n",
    "    return ds\n",
    "train['engage_time'] = train['engage_time'].map_partitions(set_nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['elapsed_time'] = train['engage_time'] - train['timestamp']\n",
    "train['elapsed_time'] = train.elapsed_time.astype('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0 603956.0\n",
      "15581.699535245267\n"
     ]
    }
   ],
   "source": [
    "print(train['elapsed_time'].min().compute(),train['elapsed_time'].max().compute())\n",
    "print(train['elapsed_time'].mean().compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>dt_minute</th>\n",
       "      <th>dt_second</th>\n",
       "      <th>elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1.581240e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>14326</td>\n",
       "      <td>408</td>\n",
       "      <td>False</td>\n",
       "      <td>2018-03-31 21:20:17</td>\n",
       "      <td>2022773</td>\n",
       "      <td>27428</td>\n",
       "      <td>600</td>\n",
       "      <td>False</td>\n",
       "      <td>2018-03-13 13:47:49</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>26</td>\n",
       "      <td>50</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1.581274e+09</td>\n",
       "      <td>10</td>\n",
       "      <td>237126</td>\n",
       "      <td>1193</td>\n",
       "      <td>True</td>\n",
       "      <td>2010-11-12 21:39:20</td>\n",
       "      <td>15871988</td>\n",
       "      <td>420</td>\n",
       "      <td>518</td>\n",
       "      <td>False</td>\n",
       "      <td>2011-09-05 16:42:09</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>41</td>\n",
       "      <td>35</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.581211e+09</td>\n",
       "      <td>19</td>\n",
       "      <td>23079</td>\n",
       "      <td>1803</td>\n",
       "      <td>False</td>\n",
       "      <td>2010-07-10 21:39:50</td>\n",
       "      <td>10982964</td>\n",
       "      <td>134</td>\n",
       "      <td>408</td>\n",
       "      <td>False</td>\n",
       "      <td>2016-05-19 02:19:01</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>28</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1.581078e+09</td>\n",
       "      <td>29</td>\n",
       "      <td>769176</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "      <td>2009-12-18 14:28:33</td>\n",
       "      <td>15871991</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>False</td>\n",
       "      <td>2019-09-10 09:17:08</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581079e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>1287.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1.581171e+09</td>\n",
       "      <td>35</td>\n",
       "      <td>73952</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "      <td>2016-06-21 21:45:25</td>\n",
       "      <td>15871992</td>\n",
       "      <td>2</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>2019-12-11 15:38:45</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581255e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>39</td>\n",
       "      <td>83948.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id  \\\n",
       "0         0      7        0           1        54  1.581240e+09          0   \n",
       "1        10      0        0           1        54  1.581274e+09         10   \n",
       "2        20      5        0           1         3  1.581211e+09         19   \n",
       "3        30      0        5           2        11  1.581078e+09         29   \n",
       "4        40      0        0           2         6  1.581171e+09         35   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  a_account_creation  \\\n",
       "0             14326                408          False 2018-03-31 21:20:17   \n",
       "1            237126               1193           True 2010-11-12 21:39:20   \n",
       "2             23079               1803          False 2010-07-10 21:39:50   \n",
       "3            769176                190          False 2009-12-18 14:28:33   \n",
       "4             73952                 13          False 2016-06-21 21:45:25   \n",
       "\n",
       "   b_user_id  b_follower_count  b_following_count  b_is_verified  \\\n",
       "0    2022773             27428                600          False   \n",
       "1   15871988               420                518          False   \n",
       "2   10982964               134                408          False   \n",
       "3   15871991                 2                 29          False   \n",
       "4   15871992                 2                 80          False   \n",
       "\n",
       "   b_account_creation  b_follows_a  reply  retweet  retweet_comment  like  \\\n",
       "0 2018-03-13 13:47:49         True    0.0      0.0              0.0   0.0   \n",
       "1 2011-09-05 16:42:09        False    0.0      0.0              0.0   0.0   \n",
       "2 2016-05-19 02:19:01        False    0.0      0.0              0.0   0.0   \n",
       "3 2019-09-10 09:17:08        False    0.0      0.0              0.0   1.0   \n",
       "4 2019-12-11 15:38:45        False    0.0      0.0              0.0   1.0   \n",
       "\n",
       "    engage_time  len_domains  len_hashtags  len_links  dt_dow  dt_hour  \\\n",
       "0           NaN            0             0          0       6        9   \n",
       "1           NaN            0             0          0       6       18   \n",
       "2           NaN            0             0          0       6        1   \n",
       "3  1.581079e+09            0             0          0       4       12   \n",
       "4  1.581255e+09            0             0          0       5       14   \n",
       "\n",
       "   dt_minute  dt_second  elapsed_time  \n",
       "0         26         50           NaN  \n",
       "1         41         35           NaN  \n",
       "2         13         28           NaN  \n",
       "3         15         20        1287.0  \n",
       "4         14         39       83948.0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23.5 ms, sys: 4.41 ms, total: 27.9 ms\n",
      "Wall time: 25.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# TRAIN FIRST 5 DAYS. VALIDATE LAST 2 DAYS\n",
    "VALID_DOW = [1, 2]# order is [3, 4, 5, 6, 0, 1, 2]\n",
    "valid = train[train['dt_dow'].isin(VALID_DOW)].reset_index(drop=True)\n",
    "train = train[~train['dt_dow'].isin(VALID_DOW)].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask.dataframe.core.DataFrame'> (Delayed('int-82428409-d948-4164-af2a-4f9dc88b991c'), 30) (Delayed('int-f62b48cd-97fb-424e-b9a4-1279e9c9afae'), 30)\n",
      "CPU times: user 33.8 ms, sys: 0 ns, total: 33.8 ms\n",
      "Wall time: 32.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "print(type(train), train.shape, valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.08 s, sys: 849 ms, total: 9.93 s\n",
      "Wall time: 1min 3s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>dt_minute</th>\n",
       "      <th>dt_second</th>\n",
       "      <th>elapsed_time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1.580947e+09</th>\n",
       "      <td>1680159</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>59</td>\n",
       "      <td>5303</td>\n",
       "      <td>1923062</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2014-05-16 22:58:30</td>\n",
       "      <td>2812169</td>\n",
       "      <td>286</td>\n",
       "      <td>808</td>\n",
       "      <td>False</td>\n",
       "      <td>2014-01-04 19:54:06</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.580947e+09</th>\n",
       "      <td>180802</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>2348</td>\n",
       "      <td>9794149</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>2011-11-14 14:57:32</td>\n",
       "      <td>22175905</td>\n",
       "      <td>114</td>\n",
       "      <td>121</td>\n",
       "      <td>False</td>\n",
       "      <td>2013-10-26 00:06:10</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.580947e+09</th>\n",
       "      <td>6569932</td>\n",
       "      <td>5</td>\n",
       "      <td>1571</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>19001</td>\n",
       "      <td>1843389</td>\n",
       "      <td>392</td>\n",
       "      <td>True</td>\n",
       "      <td>2008-08-23 13:13:44</td>\n",
       "      <td>27557184</td>\n",
       "      <td>15</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>2010-07-03 18:06:52</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.580947e+09</th>\n",
       "      <td>3552255</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>119247</td>\n",
       "      <td>1747480</td>\n",
       "      <td>1615</td>\n",
       "      <td>True</td>\n",
       "      <td>2008-06-06 22:14:20</td>\n",
       "      <td>30112618</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>2020-01-07 21:31:04</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581085e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>137335.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.580947e+09</th>\n",
       "      <td>41656685</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>14702</td>\n",
       "      <td>114453</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2012-12-22 08:53:53</td>\n",
       "      <td>24020411</td>\n",
       "      <td>111</td>\n",
       "      <td>82</td>\n",
       "      <td>False</td>\n",
       "      <td>2019-11-22 01:03:13</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id  media  domains  tweet_type  language  a_user_id  \\\n",
       "timestamp                                                                 \n",
       "1.580947e+09   1680159      5        0           2        59       5303   \n",
       "1.580947e+09    180802      5        0           2        54       2348   \n",
       "1.580947e+09   6569932      5     1571           2        54      19001   \n",
       "1.580947e+09   3552255      9        0           2         3     119247   \n",
       "1.580947e+09  41656685      9        0           2        47      14702   \n",
       "\n",
       "              a_follower_count  a_following_count  a_is_verified  \\\n",
       "timestamp                                                          \n",
       "1.580947e+09           1923062                  0          False   \n",
       "1.580947e+09           9794149                 90           True   \n",
       "1.580947e+09           1843389                392           True   \n",
       "1.580947e+09           1747480               1615           True   \n",
       "1.580947e+09            114453                  0          False   \n",
       "\n",
       "              a_account_creation  b_user_id  b_follower_count  \\\n",
       "timestamp                                                       \n",
       "1.580947e+09 2014-05-16 22:58:30    2812169               286   \n",
       "1.580947e+09 2011-11-14 14:57:32   22175905               114   \n",
       "1.580947e+09 2008-08-23 13:13:44   27557184                15   \n",
       "1.580947e+09 2008-06-06 22:14:20   30112618                 1   \n",
       "1.580947e+09 2012-12-22 08:53:53   24020411               111   \n",
       "\n",
       "              b_following_count  b_is_verified  b_account_creation  \\\n",
       "timestamp                                                            \n",
       "1.580947e+09                808          False 2014-01-04 19:54:06   \n",
       "1.580947e+09                121          False 2013-10-26 00:06:10   \n",
       "1.580947e+09                116          False 2010-07-03 18:06:52   \n",
       "1.580947e+09                 69          False 2020-01-07 21:31:04   \n",
       "1.580947e+09                 82          False 2019-11-22 01:03:13   \n",
       "\n",
       "              b_follows_a  reply  retweet  retweet_comment  like  \\\n",
       "timestamp                                                          \n",
       "1.580947e+09        False    0.0      0.0              0.0   0.0   \n",
       "1.580947e+09        False    0.0      0.0              0.0   0.0   \n",
       "1.580947e+09        False    0.0      0.0              0.0   0.0   \n",
       "1.580947e+09        False    0.0      0.0              0.0   1.0   \n",
       "1.580947e+09        False    0.0      0.0              0.0   0.0   \n",
       "\n",
       "               engage_time  len_domains  len_hashtags  len_links  dt_dow  \\\n",
       "timestamp                                                                  \n",
       "1.580947e+09           NaN            0             0          0       3   \n",
       "1.580947e+09           NaN            0             0          0       3   \n",
       "1.580947e+09           NaN            0             0          0       3   \n",
       "1.580947e+09  1.581085e+09            0             0          0       3   \n",
       "1.580947e+09           NaN            0             0          0       3   \n",
       "\n",
       "              dt_hour  dt_minute  dt_second  elapsed_time  \n",
       "timestamp                                                  \n",
       "1.580947e+09        0          0          0           NaN  \n",
       "1.580947e+09        0          0          0           NaN  \n",
       "1.580947e+09        0          0          0           NaN  \n",
       "1.580947e+09        0          0          0      137335.0  \n",
       "1.580947e+09        0          0          0           NaN  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = train.set_index('timestamp')\n",
    "valid = valid.set_index('timestamp')\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 233 ms, sys: 18.3 ms, total: 251 ms\n",
      "Wall time: 931 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>a_account_creation</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>dt_minute</th>\n",
       "      <th>dt_second</th>\n",
       "      <th>elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>1680159</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>59</td>\n",
       "      <td>5303</td>\n",
       "      <td>1923062</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2014-05-16 22:58:30</td>\n",
       "      <td>2812169</td>\n",
       "      <td>286</td>\n",
       "      <td>808</td>\n",
       "      <td>False</td>\n",
       "      <td>2014-01-04 19:54:06</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>180802</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>2348</td>\n",
       "      <td>9794149</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>2011-11-14 14:57:32</td>\n",
       "      <td>22175905</td>\n",
       "      <td>114</td>\n",
       "      <td>121</td>\n",
       "      <td>False</td>\n",
       "      <td>2013-10-26 00:06:10</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>6569932</td>\n",
       "      <td>5</td>\n",
       "      <td>1571</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>19001</td>\n",
       "      <td>1843389</td>\n",
       "      <td>392</td>\n",
       "      <td>True</td>\n",
       "      <td>2008-08-23 13:13:44</td>\n",
       "      <td>27557184</td>\n",
       "      <td>15</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>2010-07-03 18:06:52</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>3552255</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>119247</td>\n",
       "      <td>1747480</td>\n",
       "      <td>1615</td>\n",
       "      <td>True</td>\n",
       "      <td>2008-06-06 22:14:20</td>\n",
       "      <td>30112618</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>2020-01-07 21:31:04</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581085e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>137335.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>41656685</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>14702</td>\n",
       "      <td>114453</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2012-12-22 08:53:53</td>\n",
       "      <td>24020411</td>\n",
       "      <td>111</td>\n",
       "      <td>82</td>\n",
       "      <td>False</td>\n",
       "      <td>2019-11-22 01:03:13</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      timestamp  tweet_id  media  domains  tweet_type  language  a_user_id  \\\n",
       "0  1.580947e+09   1680159      5        0           2        59       5303   \n",
       "1  1.580947e+09    180802      5        0           2        54       2348   \n",
       "2  1.580947e+09   6569932      5     1571           2        54      19001   \n",
       "3  1.580947e+09   3552255      9        0           2         3     119247   \n",
       "4  1.580947e+09  41656685      9        0           2        47      14702   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  a_account_creation  \\\n",
       "0           1923062                  0          False 2014-05-16 22:58:30   \n",
       "1           9794149                 90           True 2011-11-14 14:57:32   \n",
       "2           1843389                392           True 2008-08-23 13:13:44   \n",
       "3           1747480               1615           True 2008-06-06 22:14:20   \n",
       "4            114453                  0          False 2012-12-22 08:53:53   \n",
       "\n",
       "   b_user_id  b_follower_count  b_following_count  b_is_verified  \\\n",
       "0    2812169               286                808          False   \n",
       "1   22175905               114                121          False   \n",
       "2   27557184                15                116          False   \n",
       "3   30112618                 1                 69          False   \n",
       "4   24020411               111                 82          False   \n",
       "\n",
       "   b_account_creation  b_follows_a  reply  retweet  retweet_comment  like  \\\n",
       "0 2014-01-04 19:54:06        False    0.0      0.0              0.0   0.0   \n",
       "1 2013-10-26 00:06:10        False    0.0      0.0              0.0   0.0   \n",
       "2 2010-07-03 18:06:52        False    0.0      0.0              0.0   0.0   \n",
       "3 2020-01-07 21:31:04        False    0.0      0.0              0.0   1.0   \n",
       "4 2019-11-22 01:03:13        False    0.0      0.0              0.0   0.0   \n",
       "\n",
       "    engage_time  len_domains  len_hashtags  len_links  dt_dow  dt_hour  \\\n",
       "0           NaN            0             0          0       3        0   \n",
       "1           NaN            0             0          0       3        0   \n",
       "2           NaN            0             0          0       3        0   \n",
       "3  1.581085e+09            0             0          0       3        0   \n",
       "4           NaN            0             0          0       3        0   \n",
       "\n",
       "   dt_minute  dt_second  elapsed_time  \n",
       "0          0          0           NaN  \n",
       "1          0          0           NaN  \n",
       "2          0          0           NaN  \n",
       "3          0          0      137335.0  \n",
       "4          0          0           NaN  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = train.reset_index()\n",
    "valid = valid.reset_index()\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for i,p in enumerate(train.partitions):\n",
    "#    print(i,len(p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for i,p in enumerate(valid.partitions):\n",
    "#    print(i,len(p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MTE_one_shot:\n",
    "    \n",
    "    def __init__(self, folds, smooth, seed=42):\n",
    "        self.folds = folds\n",
    "        self.seed = seed\n",
    "        self.smooth = smooth\n",
    "        \n",
    "    def fit_transform(self, train, x_col, y_col, y_mean=None, out_col = None, out_dtype=None):\n",
    "        \n",
    "        self.y_col = y_col\n",
    "        np.random.seed(self.seed)\n",
    "        \n",
    "        if 'fold' not in train.columns:\n",
    "            fsize = len(train)//self.folds\n",
    "            train['fold'] = 1\n",
    "            train['fold'] = train['fold'].cumsum()\n",
    "            train['fold'] = train['fold']//fsize\n",
    "            train['fold'] = train['fold']%self.folds\n",
    "        \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'TE_{tag}_{self.y_col}'\n",
    "        \n",
    "        if y_mean is None:\n",
    "            y_mean = train[y_col].mean()#.compute().astype('float32')\n",
    "        self.mean = y_mean\n",
    "        \n",
    "        cols = ['fold',x_col] if isinstance(x_col,str) else ['fold']+x_col\n",
    "        \n",
    "        agg_each_fold = train.groupby(cols).agg({y_col:['count','sum']}).reset_index()\n",
    "        agg_each_fold.columns = cols + ['count_y','sum_y']\n",
    "        \n",
    "        agg_all = agg_each_fold.groupby(x_col).agg({'count_y':'sum','sum_y':'sum'}).reset_index()\n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all.columns = cols + ['count_y_all','sum_y_all']\n",
    "        \n",
    "        agg_each_fold = agg_each_fold.merge(agg_all,on=x_col,how='left')\n",
    "        agg_each_fold['count_y_all'] = agg_each_fold['count_y_all'] - agg_each_fold['count_y']\n",
    "        agg_each_fold['sum_y_all'] = agg_each_fold['sum_y_all'] - agg_each_fold['sum_y']\n",
    "        agg_each_fold[out_col] = (agg_each_fold['sum_y_all']+self.smooth*self.mean)/(agg_each_fold['count_y_all']+self.smooth)\n",
    "        agg_each_fold = agg_each_fold.drop(['count_y_all','count_y','sum_y_all','sum_y'],axis=1)\n",
    "        \n",
    "        agg_all[out_col] = (agg_all['sum_y_all']+self.smooth*self.mean)/(agg_all['count_y_all']+self.smooth)\n",
    "        agg_all = agg_all.drop(['count_y_all','sum_y_all'],axis=1)\n",
    "        self.agg_all = agg_all\n",
    "        \n",
    "        train.columns\n",
    "        cols = ['fold',x_col] if isinstance(x_col,str) else ['fold']+x_col\n",
    "        train = train.merge(agg_each_fold,on=cols,how='left')\n",
    "        del agg_each_fold\n",
    "        #self.agg_each_fold = agg_each_fold\n",
    "        #train[out_col] = train.map_partitions(lambda cudf_df: cudf_df[out_col].nans_to_nulls())\n",
    "        train[out_col] = train[out_col].fillna(self.mean)\n",
    "        \n",
    "        if out_dtype is not None:\n",
    "            train[out_col] = train[out_col].astype(out_dtype)\n",
    "        return train\n",
    "    \n",
    "    def transform(self, test, x_col, out_col = None, out_dtype=None):\n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'TE_{tag}_{self.y_col}'\n",
    "        test = test.merge(self.agg_all,on=x_col,how='left')\n",
    "        test[out_col] = test[out_col].fillna(self.mean)\n",
    "        if out_dtype is not None:\n",
    "            test[out_col] = test[out_col].astype(out_dtype)\n",
    "        return test\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TE_media_reply 17.8 seconds<br>\n",
    "TE_tweet_type_reply 27.1 seconds<br>\n",
    "TE_language_reply 52.5 seconds<br>\n",
    "TE_a_user_id_reply 180.0 seconds<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TE_media_reply 27.4 seconds\n",
      "TE_tweet_type_reply 45.6 seconds\n",
      "TE_language_reply 64.2 seconds\n",
      "TE_a_user_id_reply 151.4 seconds\n",
      "TE_b_user_id_reply 351.9 seconds\n",
      "TE_media_retweet 27.0 seconds\n",
      "TE_tweet_type_retweet 54.8 seconds\n",
      "TE_language_retweet 83.1 seconds\n",
      "TE_a_user_id_retweet 181.7 seconds\n",
      "TE_b_user_id_retweet 392.7 seconds\n",
      "TE_media_retweet_comment 29.5 seconds\n",
      "TE_tweet_type_retweet_comment 59.2 seconds\n",
      "TE_language_retweet_comment 89.3 seconds\n",
      "TE_a_user_id_retweet_comment 190.3 seconds\n",
      "TE_b_user_id_retweet_comment 401.2 seconds\n",
      "TE_media_like 27.0 seconds\n",
      "TE_tweet_type_like 52.0 seconds\n",
      "TE_language_like 77.5 seconds\n",
      "TE_a_user_id_like 163.9 seconds\n",
      "TE_b_user_id_like 373.0 seconds\n",
      "CPU times: user 3min 55s, sys: 17.1 s, total: 4min 12s\n",
      "Wall time: 25min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "start = time.time()\n",
    "for t in ['reply', 'retweet', 'retweet_comment', 'like']:\n",
    "    start = time.time()\n",
    "    for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "        out_col = f'TE_{c}_{t}'\n",
    "        encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "        train = encoder.fit_transform(train, c, t, out_col=out_col, out_dtype='float32')\n",
    "        valid = encoder.transform(valid, c, out_col=out_col, out_dtype='float32')\n",
    "        cols.append(out_col)\n",
    "        train,valid = dask.persist(train,valid)\n",
    "        del encoder\n",
    "        #train.head()\n",
    "        wait(train)\n",
    "        wait(valid)\n",
    "        print(out_col,\"%.1f seconds\"%(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    21447306\n",
       "4    21447304\n",
       "3    21447304\n",
       "2    21447304\n",
       "1    21447304\n",
       "Name: fold, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['fold'].value_counts().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiple Column Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.04 s, sys: 68.2 ms, total: 1.1 s\n",
      "Wall time: 1.04 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols=[]\n",
    "c = ['domains','language','b_follows_a','tweet_type','media','a_is_verified']\n",
    "for t in ['reply', 'retweet', 'retweet_comment', 'like']:\n",
    "    out_col = f'TE_multi_{t}'\n",
    "    encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "    train = encoder.fit_transform(train, c, t, out_col=out_col, out_dtype='float32')\n",
    "    valid = encoder.transform(valid, c, out_col=out_col, out_dtype='float32')\n",
    "    cols.append(out_col)\n",
    "    del encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30.6 s, sys: 2.08 s, total: 32.7 s\n",
      "Wall time: 2min 59s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DoneAndNotDoneFutures(done={<Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 7)>, <Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 6)>, <Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 5)>, <Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 4)>, <Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 3)>, <Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 2)>, <Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 0)>, <Future: finished, type: pandas.DataFrame, key: ('assign-a50e363cfce36e43c6ce3e38ded602c0', 1)>}, not_done=set())"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "wait(train)\n",
    "wait(valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elapsed Time Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TE_media_elapsed_time 0.2 seconds\n",
      "TE_tweet_type_elapsed_time 0.4 seconds\n",
      "TE_language_elapsed_time 0.6 seconds\n",
      "CPU times: user 578 ms, sys: 20.9 ms, total: 599 ms\n",
      "Wall time: 566 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "start = time.time()\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language']:#, 'a_user_id', 'b_user_id']:\n",
    "    for t in ['elapsed_time']:\n",
    "        out_col = f'TE_{c}_{t}'\n",
    "        encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "        train = encoder.fit_transform(train, c, t, out_col=out_col)\n",
    "        out_dtype='float32' #if 'user_id' in c else None\n",
    "        valid = encoder.transform(valid, c, out_col=out_col, out_dtype=out_dtype)\n",
    "        cols.append(out_col)\n",
    "        print(out_col,\"%.1f seconds\"%(time.time()-start))\n",
    "        #del encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15.9 s, sys: 1.03 s, total: 17 s\n",
      "Wall time: 1min 28s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DoneAndNotDoneFutures(done={<Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 4)>, <Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 2)>, <Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 0)>, <Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 5)>, <Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 3)>, <Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 7)>, <Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 6)>, <Future: finished, type: pandas.DataFrame, key: ('assign-400c6aa823c4197d0c704a041c68d56a', 1)>}, not_done=set())"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "wait(train)\n",
    "wait(valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FrequencyEncoder:\n",
    "    \n",
    "    def __init__(self, seed=42):\n",
    "        self.seed = seed\n",
    "        \n",
    "    def fit_transform(self, train, x_col, c_col=None, out_col = None):\n",
    "        np.random.seed(self.seed)\n",
    "        if c_col is None or c_col not in train.columns:\n",
    "            c_col = 'dummy'\n",
    "            train[c_col] = 1\n",
    "            drop = True\n",
    "        else:\n",
    "            drop = False\n",
    "            \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'CE_{tag}_norm'\n",
    "            \n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all = train.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        if drop:\n",
    "            train = train.drop(c_col,axis=1)\n",
    "        agg_all.columns = cols + [out_col]\n",
    "        agg_all[out_col] = agg_all[out_col].astype('int32')\n",
    "        agg_all[out_col] = agg_all[out_col]*1.0/len(train)\n",
    "        agg_all[out_col] = agg_all[out_col].astype('float32')\n",
    "    \n",
    "        train = train.merge(agg_all,on=cols,how='left')\n",
    "        del agg_all\n",
    "        #print(train.columns)\n",
    "        #train[out_col] = train.map_partitions(lambda cudf_df: cudf_df[out_col].nans_to_nulls())\n",
    "        return train\n",
    "    \n",
    "    def transform(self, test, x_col, c_col=None, out_col = None):\n",
    "        return self.fit_transform(test, x_col, c_col, out_col)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CountEncoder:\n",
    "    \n",
    "    def __init__(self, seed=42):\n",
    "        self.seed = seed\n",
    "        \n",
    "    def fit_transform(self, train, test, x_col, out_col = None):\n",
    "        np.random.seed(self.seed)\n",
    "        \n",
    "        common_cols = [i for i in train.columns if i in test.columns and i!=x_col]\n",
    "\n",
    "        if len(common_cols):\n",
    "            c_col = common_cols[0]\n",
    "            drop = False\n",
    "        else:\n",
    "            c_col = 'dummy'\n",
    "            train[c_col] = 1\n",
    "            test[c_col]=1\n",
    "            drop = True\n",
    "            \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'CE_{tag}_norm'\n",
    "            \n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all = train.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        agg_all.columns = cols + [out_col]\n",
    "        \n",
    "        agg_test = test.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        agg_test.columns = cols + [out_col+'_test']\n",
    "        agg_all = agg_all.merge(agg_test,on=cols,how='left')\n",
    "        agg_all[out_col+'_test'] = agg_all[out_col+'_test'].fillna(0)\n",
    "        agg_all[out_col] = agg_all[out_col] + agg_all[out_col+'_test']\n",
    "        agg_all = agg_all.drop(out_col+'_test', axis=1)\n",
    "        del agg_test\n",
    "            \n",
    "        if drop:\n",
    "            train = train.drop(c_col,axis=1)\n",
    "            test = test.drop(c_col,axis=1)\n",
    "        train = train.merge(agg_all,on=cols,how='left')\n",
    "        test = test.merge(agg_all,on=cols,how='left')\n",
    "        del agg_all\n",
    "        return train,test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CE_media 20.8 seconds\n",
      "CE_tweet_type 48.1 seconds\n",
      "CE_language 76.7 seconds\n",
      "CE_a_user_id 132.1 seconds\n",
      "CE_b_user_id 223.1 seconds\n",
      "CPU times: user 38.5 s, sys: 2.19 s, total: 40.7 s\n",
      "Wall time: 3min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF CE ENCODING IS SUPER FAST!!\n",
    "start = time.time()\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "    encoder = CountEncoder()\n",
    "    out_col = f'CE_{c}'\n",
    "    train,valid = encoder.fit_transform(train, valid, c, out_col=out_col)\n",
    "    print\n",
    "    del encoder\n",
    "    train,valid = dask.persist(train,valid)\n",
    "    wait(train)\n",
    "    wait(valid)\n",
    "    print(out_col,\"%.1f seconds\"%(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CE_media_norm 42.8 seconds\n",
      "CE_tweet_type_norm 78.8 seconds\n",
      "CE_language_norm 118.8 seconds\n",
      "CE_a_user_id_norm 174.3 seconds\n",
      "CE_b_user_id_norm 262.9 seconds\n",
      "CPU times: user 45.2 s, sys: 2.71 s, total: 47.9 s\n",
      "Wall time: 4min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF CE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "start = time.time()\n",
    "for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "    encoder = FrequencyEncoder()\n",
    "    out_col = f'CE_{c}_norm'\n",
    "    train = encoder.fit_transform(train, c, c_col='tweet_id', out_col=out_col)\n",
    "    valid = encoder.transform(valid, c, c_col='tweet_id', out_col=out_col)\n",
    "    cols.append(out_col)\n",
    "    del encoder\n",
    "    train,valid = dask.persist(train,valid)\n",
    "    wait(train)\n",
    "    wait(valid)\n",
    "    print(out_col,\"%.1f seconds\"%(time.time()-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Difference Encode (Lag Features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_encode_cudf_v1(train,col,tar,sft=1):\n",
    "    train[col+'_sft'] = train[col].shift(sft)\n",
    "    train[tar+'_sft'] = train[tar].shift(sft)\n",
    "    out_col = f'DE_{col}_{tar}_{sft}'\n",
    "    train[out_col] = train[tar]-train[tar+'_sft']\n",
    "    mask = '__MASK__'\n",
    "    train[mask] = train[col] == train[col+'_sft']\n",
    "    train = train.drop([col+'_sft',tar+'_sft'],axis=1)\n",
    "    train[out_col] = train[out_col]*train[mask]\n",
    "    train = train.drop(mask,axis=1)\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DE b_user_id b_follower_count 1 22.2 seconds\n",
      "DE b_user_id b_follower_count -1 21.7 seconds\n",
      "DE b_user_id b_following_count 1 20.4 seconds\n",
      "DE b_user_id b_following_count -1 19.5 seconds\n",
      "DE b_user_id language 1 18.1 seconds\n",
      "DE b_user_id language -1 19.4 seconds\n",
      "CPU times: user 24.7 s, sys: 1.51 s, total: 26.3 s\n",
      "Wall time: 2min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "start = time.time()\n",
    "# cuDF DE ENCODING IS FAST!!\n",
    "idx = 0; cols = []; sc = 'timestamp'\n",
    "for c in ['b_user_id']:\n",
    "    for t in ['b_follower_count','b_following_count','language']:\n",
    "        for s in [1,-1]:\n",
    "            start = time.time()\n",
    "            train = diff_encode_cudf_v1(train, col=c, tar=t, sft=s)\n",
    "            valid = diff_encode_cudf_v1(valid, col=c, tar=t, sft=s)\n",
    "            train,valid = dask.persist(train,valid)\n",
    "            wait(train)\n",
    "            wait(valid)\n",
    "            end = time.time(); idx += 1\n",
    "            print('DE',c,t,s,'%.1f seconds'%(end-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Diff Language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_lang = train[['a_user_id', 'language', 'tweet_id']].drop_duplicates()\n",
    "valid_lang = valid[['a_user_id', 'language', 'tweet_id']].drop_duplicates()\n",
    "train_lang_count = train_lang.groupby(['a_user_id', 'language']).agg({'tweet_id':'count'}).reset_index()\n",
    "valid_lang_count = valid_lang.groupby(['a_user_id', 'language']).agg({'tweet_id':'count'}).reset_index()\n",
    "train_lang_count,valid_lang_count = dask.persist(train_lang_count,valid_lang_count)\n",
    "train_lang_count.head()\n",
    "del train_lang,valid_lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.5 s, sys: 80.1 ms, total: 1.58 s\n",
      "Wall time: 9.63 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>top_language</th>\n",
       "      <th>language_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a_user_id  top_language  language_count\n",
       "0          0             3             3.0\n",
       "1          0             4             1.0\n",
       "2          0            11            21.0\n",
       "3          0            13             3.0\n",
       "4          0            38             1.0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train_lang_count = train_lang_count.merge(valid_lang_count,on=['a_user_id', 'language'],how='left')\n",
    "train_lang_count['tweet_id_y'] = train_lang_count['tweet_id_y'].fillna(0)\n",
    "train_lang_count['tweet_id_x'] = train_lang_count['tweet_id_x'] + train_lang_count['tweet_id_y']\n",
    "train_lang_count = train_lang_count.drop('tweet_id_y',axis=1)\n",
    "train_lang_count.columns = ['a_user_id', 'top_language', 'language_count']\n",
    "train_lang_count, = dask.persist(train_lang_count)\n",
    "train_lang_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'sort_values'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rapids/lib/python3.7/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3442\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3443\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3444\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"'DataFrame' object has no attribute %r\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3445\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3446\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__dir__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'sort_values'"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_lang_count = train_lang_count.sort_values(['a_user_id', 'language_count'])\n",
    "train_lang_count['a_user_shifted'] = train_lang_count['a_user_id'].shift(1)\n",
    "train_lang_count = train_lang_count[train_lang_count['a_user_id']!=train_lang_count['a_user_shifted']]\n",
    "train_lang_count = train_lang_count.drop(['a_user_shifted','language_count'],axis=1)\n",
    "train_lang_count.columns = ['a_user_id','top_language']\n",
    "train_lang_count, = dask.persist(train_lang_count)\n",
    "train_lang_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_language(df,df_lang_count):\n",
    "    df = df.merge(df_lang_count,how='left', left_on='b_user_id', right_on='a_user_id')\n",
    "    df['nan_language'] = df['top_language'].isnull()\n",
    "    df['same_language'] = df['language'] == df['top_language']\n",
    "    df['diff_language'] = df['language'] != df['top_language']\n",
    "    df['same_language'] = df['same_language']*(1-df['nan_language'])\n",
    "    df['diff_language'] = df['diff_language']*(1-df['nan_language'])\n",
    "    df = df.drop('top_language',axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#train = diff_language(train,train_lang_count)\n",
    "#valid = diff_language(valid,train_lang_count)\n",
    "#train,valid = dask.persist(train,valid)\n",
    "#train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rate feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 65.6 ms, sys: 3.93 ms, total: 69.6 ms\n",
      "Wall time: 66.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# follow rate feature\n",
    "train['a_ff_rate'] = (train['a_following_count'] / train['a_follower_count']).astype('float32')\n",
    "train['b_ff_rate'] = (train['b_follower_count']  / train['b_following_count']).astype('float32')\n",
    "valid['a_ff_rate']  = (valid['a_following_count'] / valid['a_follower_count']).astype('float32')\n",
    "valid['b_ff_rate']  = (valid['b_follower_count']  / valid['b_following_count']).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "train,valid = dask.persist(train,valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DoneAndNotDoneFutures(done={<Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 3)>, <Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 7)>, <Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 1)>, <Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 6)>, <Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 2)>, <Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 4)>, <Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 5)>, <Future: finished, type: pandas.DataFrame, key: ('assign-f50bd9d59f87b205508fbd3053c135ef', 0)>}, not_done=set())"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wait(train)\n",
    "wait(valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarize Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 204 µs, sys: 0 ns, total: 204 µs\n",
      "Wall time: 211 µs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['tweet_id',\n",
       " 'timestamp',\n",
       " 'a_account_creation',\n",
       " 'b_account_creation',\n",
       " 'engage_time',\n",
       " 'fold',\n",
       " 'b_user_id',\n",
       " 'a_user_id',\n",
       " 'dt_dow',\n",
       " 'a_account_creation',\n",
       " 'b_account_creation',\n",
       " 'elapsed_time',\n",
       " 'domains']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "label_names = ['reply', 'retweet', 'retweet_comment', 'like']\n",
    "DONT_USE = ['tweet_id','timestamp','a_account_creation','b_account_creation','engage_time',\n",
    "            'fold','b_user_id','a_user_id', 'dt_dow',\n",
    "            'a_account_creation', 'b_account_creation', 'elapsed_time',\n",
    "             'links','domains','hashtags0','hashtags1']\n",
    "DONT_USE += label_names\n",
    "features = [c for c in train.columns if c not in DONT_USE]\n",
    "\n",
    "RMV = [c for c in DONT_USE if c in train.columns and c not in label_names]\n",
    "RMV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.75 s, sys: 191 ms, total: 3.94 s\n",
      "Wall time: 16 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for col in RMV:\n",
    "    #print(col, col in train.columns)\n",
    "    if col in train.columns:\n",
    "        train = train.drop(col,axis=1)\n",
    "        train, = dask.persist(train)\n",
    "        train.head()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.22 s, sys: 190 ms, total: 3.41 s\n",
      "Wall time: 13.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for col in RMV:\n",
    "    #print(col, col in valid.columns)\n",
    "    if col in valid.columns:\n",
    "        valid = valid.drop(col,axis=1)\n",
    "        valid, = dask.persist(valid,)\n",
    "        valid.head()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model Validate\n",
    "We will train on random 10% of first 5 days and validation on last 2 days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107236522\n",
      "10723651\n",
      "Using 61 features: 61\n",
      "CPU times: user 8.9 s, sys: 8.65 s, total: 17.6 s\n",
      "Wall time: 22 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['media', 'tweet_type', 'language', 'a_follower_count',\n",
       "       'a_following_count', 'a_is_verified', 'b_follower_count',\n",
       "       'b_following_count', 'b_is_verified', 'b_follows_a', 'len_domains',\n",
       "       'len_hashtags', 'len_links', 'dt_hour', 'dt_minute', 'dt_second',\n",
       "       'TE_media_reply', 'TE_tweet_type_reply', 'TE_language_reply',\n",
       "       'TE_a_user_id_reply', 'TE_b_user_id_reply', 'TE_media_retweet',\n",
       "       'TE_tweet_type_retweet', 'TE_language_retweet',\n",
       "       'TE_a_user_id_retweet', 'TE_b_user_id_retweet',\n",
       "       'TE_media_retweet_comment', 'TE_tweet_type_retweet_comment',\n",
       "       'TE_language_retweet_comment', 'TE_a_user_id_retweet_comment',\n",
       "       'TE_b_user_id_retweet_comment', 'TE_media_like',\n",
       "       'TE_tweet_type_like', 'TE_language_like', 'TE_a_user_id_like',\n",
       "       'TE_b_user_id_like', 'TE_multi_reply', 'TE_multi_retweet',\n",
       "       'TE_multi_retweet_comment', 'TE_multi_like',\n",
       "       'TE_media_elapsed_time', 'TE_tweet_type_elapsed_time',\n",
       "       'TE_language_elapsed_time', 'CE_media', 'CE_tweet_type',\n",
       "       'CE_language', 'CE_a_user_id', 'CE_b_user_id', 'CE_media_norm',\n",
       "       'CE_tweet_type_norm', 'CE_language_norm', 'CE_a_user_id_norm',\n",
       "       'CE_b_user_id_norm', 'DE_b_user_id_b_follower_count_1',\n",
       "       'DE_b_user_id_b_follower_count_-1',\n",
       "       'DE_b_user_id_b_following_count_1',\n",
       "       'DE_b_user_id_b_following_count_-1', 'DE_b_user_id_language_1',\n",
       "       'DE_b_user_id_language_-1', 'a_ff_rate', 'b_ff_rate'], dtype='<U33')"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "SAMPLE_RATIO = 0.1\n",
    "SEED = 1\n",
    "\n",
    "if SAMPLE_RATIO < 1.0:\n",
    "    print(len(train))\n",
    "    train = train.sample(frac=SAMPLE_RATIO,random_state=42)\n",
    "    train, = dask.persist(train)\n",
    "    train.head()\n",
    "    print(len(train))\n",
    "\n",
    "train = train.compute()\n",
    "Y_train = train[label_names]\n",
    "train = train.drop(label_names,axis=1)\n",
    "\n",
    "features = [c for c in train.columns if c not in DONT_USE]\n",
    "print('Using %i features:'%(len(features)),train.shape[1])\n",
    "np.asarray(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40838716\n",
      "14293552\n"
     ]
    }
   ],
   "source": [
    "SAMPLE_RATIO = 0.35 # VAL SET NOW SIZE OF TEST SET\n",
    "SEED = 1\n",
    "if SAMPLE_RATIO < 1.0:\n",
    "    print(len(valid))\n",
    "    valid = valid.sample(frac=SAMPLE_RATIO,random_state=42)\n",
    "    valid, = dask.persist(valid)\n",
    "    valid.head()\n",
    "    print(len(valid))\n",
    "    \n",
    "valid = valid.compute()\n",
    "Y_valid = valid[label_names]\n",
    "valid = valid.drop(label_names,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGB Version 1.2.0-SNAPSHOT\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "print('XGB Version',xgb.__version__)\n",
    "\n",
    "xgb_parms = { \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.1, \n",
    "    'subsample':0.8,\n",
    "    'colsample_bytree':0.3, \n",
    "    'eval_metric':'logloss',\n",
    "    'objective':'binary:logistic',\n",
    "    'nthread':40,\n",
    "    'tree_method':'hist',\n",
    "    #'predictor' : 'gpu_predictor'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no dup :) \n",
      "X_train.shape (10723651, 61)\n",
      "X_valid.shape (14293552, 61)\n"
     ]
    }
   ],
   "source": [
    "if train.columns.duplicated().sum()>0:\n",
    "    raise Exception(f'duplicated!: { train.columns[train.columns.duplicated()] }')\n",
    "print('no dup :) ')\n",
    "print(f'X_train.shape {train.shape}')\n",
    "print(f'X_valid.shape {valid.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 52.5 ms, sys: 12.1 ms, total: 64.6 ms\n",
      "Wall time: 57.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for col in train.columns:\n",
    "    if train[col].dtype=='bool':\n",
    "        train[col] = train[col].astype('int8')\n",
    "        valid[col] = valid[col].astype('int8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################\n",
      "### reply\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 34.9 seconds\n",
      "Training...\n",
      "Took 158.8 seconds\n",
      "Predicting...\n",
      "Took 17.8 seconds\n",
      "#########################\n",
      "### retweet\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 34.4 seconds\n",
      "Training...\n",
      "Took 159.8 seconds\n",
      "Predicting...\n",
      "Took 17.3 seconds\n",
      "#########################\n",
      "### retweet_comment\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 34.9 seconds\n",
      "Training...\n",
      "Took 154.9 seconds\n",
      "Predicting...\n",
      "Took 17.6 seconds\n",
      "#########################\n",
      "### like\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 34.8 seconds\n",
      "Training...\n",
      "Took 159.1 seconds\n",
      "Predicting...\n",
      "Took 18.1 seconds\n",
      "CPU times: user 7h 30min 58s, sys: 9min 14s, total: 7h 40min 12s\n",
      "Wall time: 14min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# TRAIN AND VALIDATE\n",
    "\n",
    "NROUND = 300\n",
    "VERBOSE_EVAL = 50\n",
    "#ESR = 50\n",
    "    \n",
    "oof = np.zeros((len(valid),len(label_names)))\n",
    "preds = []\n",
    "for i in range(4):\n",
    "\n",
    "    name = label_names[i]\n",
    "    print('#'*25);print('###',name);print('#'*25)\n",
    "       \n",
    "    start = time.time(); print('Creating DMatrix...')\n",
    "        \n",
    "    dtrain = xgb.DMatrix(data=train,label=Y_train.iloc[:, i])\n",
    "    dvalid = xgb.DMatrix(data=valid,label=Y_valid.iloc[:, i])\n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "             \n",
    "    start = time.time(); print('Training...')\n",
    "    model = xgb.train(xgb_parms, \n",
    "                           dtrain=dtrain,\n",
    "                           #evals=[(dtrain,'train'),(dvalid,'valid')],\n",
    "                           num_boost_round=NROUND,\n",
    "                           #early_stopping_rounds=ESR,\n",
    "                           verbose_eval=VERBOSE_EVAL) \n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "        \n",
    "    start = time.time(); print('Predicting...')\n",
    "    #Y_valid[f'pred_{name}'] = xgb.dask.predict(client,model,valid)\n",
    "    oof[:, i] += model.predict(dvalid)\n",
    "    #preds.append(xgb.dask.predict(client,model,valid))\n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "        \n",
    "    del model, dtrain, dvalid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "yvalid = Y_valid[label_names].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute Validation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve, auc, log_loss\n",
    "\n",
    "def compute_prauc(pred, gt):\n",
    "  prec, recall, thresh = precision_recall_curve(gt, pred)\n",
    "  prauc = auc(recall, prec)\n",
    "  return prauc\n",
    "\n",
    "def calculate_ctr(gt):\n",
    "  positive = len([x for x in gt if x == 1])\n",
    "  ctr = positive/float(len(gt))\n",
    "  return ctr\n",
    "\n",
    "def compute_rce(pred, gt):\n",
    "    cross_entropy = log_loss(gt, pred)\n",
    "    data_ctr = calculate_ctr(gt)\n",
    "    strawman_cross_entropy = log_loss(gt, [data_ctr for _ in range(len(gt))])\n",
    "    return (1.0 - cross_entropy/strawman_cross_entropy)*100.0\n",
    "\n",
    "# FAST METRIC FROM GIBA\n",
    "def compute_rce_fast(pred, gt):\n",
    "    cross_entropy = log_loss(gt, pred)\n",
    "    yt = np.mean(gt)     \n",
    "    strawman_cross_entropy = -(yt*np.log(yt) + (1 - yt)*np.log(1 - yt))\n",
    "    return (1.0 - cross_entropy/strawman_cross_entropy)*100.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reply                PRAUC:0.17641 RCE:19.16150\n",
      "retweet              PRAUC:0.53312 RCE:28.65558\n",
      "retweet_comment      PRAUC:0.05191 RCE:11.15791\n",
      "like                 PRAUC:0.78464 RCE:27.56908\n",
      "CPU times: user 38.8 s, sys: 1.57 s, total: 40.4 s\n",
      "Wall time: 34.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "txt = ''\n",
    "for i in range(4):\n",
    "    prauc = compute_prauc(oof[:,i], yvalid[:, i])\n",
    "    rce   = compute_rce_fast(oof[:,i], yvalid[:, i])\n",
    "    txt_ = f\"{label_names[i]:20} PRAUC:{prauc:.5f} RCE:{rce:.5f}\"\n",
    "    print(txt_)\n",
    "    txt += txt_ + '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook took 59.5 minutes\n"
     ]
    }
   ],
   "source": [
    "print('This notebook took %.1f minutes'%((time.time()-very_start)/60.))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
